database.py 50 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339
  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2012-2017 The Python Software Foundation.
  4. # See LICENSE.txt and CONTRIBUTORS.txt.
  5. #
  6. """PEP 376 implementation."""
  7. from __future__ import unicode_literals
  8. import base64
  9. import codecs
  10. import contextlib
  11. import hashlib
  12. import logging
  13. import os
  14. import posixpath
  15. import sys
  16. import zipimport
  17. from . import DistlibException, resources
  18. from .compat import StringIO
  19. from .version import get_scheme, UnsupportedVersionError
  20. from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME,
  21. LEGACY_METADATA_FILENAME)
  22. from .util import (parse_requirement, cached_property, parse_name_and_version,
  23. read_exports, write_exports, CSVReader, CSVWriter)
  24. __all__ = ['Distribution', 'BaseInstalledDistribution',
  25. 'InstalledDistribution', 'EggInfoDistribution',
  26. 'DistributionPath']
  27. logger = logging.getLogger(__name__)
  28. EXPORTS_FILENAME = 'pydist-exports.json'
  29. COMMANDS_FILENAME = 'pydist-commands.json'
  30. DIST_FILES = ('INSTALLER', METADATA_FILENAME, 'RECORD', 'REQUESTED',
  31. 'RESOURCES', EXPORTS_FILENAME, 'SHARED')
  32. DISTINFO_EXT = '.dist-info'
  33. class _Cache(object):
  34. """
  35. A simple cache mapping names and .dist-info paths to distributions
  36. """
  37. def __init__(self):
  38. """
  39. Initialise an instance. There is normally one for each DistributionPath.
  40. """
  41. self.name = {}
  42. self.path = {}
  43. self.generated = False
  44. def clear(self):
  45. """
  46. Clear the cache, setting it to its initial state.
  47. """
  48. self.name.clear()
  49. self.path.clear()
  50. self.generated = False
  51. def add(self, dist):
  52. """
  53. Add a distribution to the cache.
  54. :param dist: The distribution to add.
  55. """
  56. if dist.path not in self.path:
  57. self.path[dist.path] = dist
  58. self.name.setdefault(dist.key, []).append(dist)
  59. class DistributionPath(object):
  60. """
  61. Represents a set of distributions installed on a path (typically sys.path).
  62. """
  63. def __init__(self, path=None, include_egg=False):
  64. """
  65. Create an instance from a path, optionally including legacy (distutils/
  66. setuptools/distribute) distributions.
  67. :param path: The path to use, as a list of directories. If not specified,
  68. sys.path is used.
  69. :param include_egg: If True, this instance will look for and return legacy
  70. distributions as well as those based on PEP 376.
  71. """
  72. if path is None:
  73. path = sys.path
  74. self.path = path
  75. self._include_dist = True
  76. self._include_egg = include_egg
  77. self._cache = _Cache()
  78. self._cache_egg = _Cache()
  79. self._cache_enabled = True
  80. self._scheme = get_scheme('default')
  81. def _get_cache_enabled(self):
  82. return self._cache_enabled
  83. def _set_cache_enabled(self, value):
  84. self._cache_enabled = value
  85. cache_enabled = property(_get_cache_enabled, _set_cache_enabled)
  86. def clear_cache(self):
  87. """
  88. Clears the internal cache.
  89. """
  90. self._cache.clear()
  91. self._cache_egg.clear()
  92. def _yield_distributions(self):
  93. """
  94. Yield .dist-info and/or .egg(-info) distributions.
  95. """
  96. # We need to check if we've seen some resources already, because on
  97. # some Linux systems (e.g. some Debian/Ubuntu variants) there are
  98. # symlinks which alias other files in the environment.
  99. seen = set()
  100. for path in self.path:
  101. finder = resources.finder_for_path(path)
  102. if finder is None:
  103. continue
  104. r = finder.find('')
  105. if not r or not r.is_container:
  106. continue
  107. rset = sorted(r.resources)
  108. for entry in rset:
  109. r = finder.find(entry)
  110. if not r or r.path in seen:
  111. continue
  112. if self._include_dist and entry.endswith(DISTINFO_EXT):
  113. possible_filenames = [METADATA_FILENAME,
  114. WHEEL_METADATA_FILENAME,
  115. LEGACY_METADATA_FILENAME]
  116. for metadata_filename in possible_filenames:
  117. metadata_path = posixpath.join(entry, metadata_filename)
  118. pydist = finder.find(metadata_path)
  119. if pydist:
  120. break
  121. else:
  122. continue
  123. with contextlib.closing(pydist.as_stream()) as stream:
  124. metadata = Metadata(fileobj=stream, scheme='legacy')
  125. logger.debug('Found %s', r.path)
  126. seen.add(r.path)
  127. yield new_dist_class(r.path, metadata=metadata,
  128. env=self)
  129. elif self._include_egg and entry.endswith(('.egg-info',
  130. '.egg')):
  131. logger.debug('Found %s', r.path)
  132. seen.add(r.path)
  133. yield old_dist_class(r.path, self)
  134. def _generate_cache(self):
  135. """
  136. Scan the path for distributions and populate the cache with
  137. those that are found.
  138. """
  139. gen_dist = not self._cache.generated
  140. gen_egg = self._include_egg and not self._cache_egg.generated
  141. if gen_dist or gen_egg:
  142. for dist in self._yield_distributions():
  143. if isinstance(dist, InstalledDistribution):
  144. self._cache.add(dist)
  145. else:
  146. self._cache_egg.add(dist)
  147. if gen_dist:
  148. self._cache.generated = True
  149. if gen_egg:
  150. self._cache_egg.generated = True
  151. @classmethod
  152. def distinfo_dirname(cls, name, version):
  153. """
  154. The *name* and *version* parameters are converted into their
  155. filename-escaped form, i.e. any ``'-'`` characters are replaced
  156. with ``'_'`` other than the one in ``'dist-info'`` and the one
  157. separating the name from the version number.
  158. :parameter name: is converted to a standard distribution name by replacing
  159. any runs of non- alphanumeric characters with a single
  160. ``'-'``.
  161. :type name: string
  162. :parameter version: is converted to a standard version string. Spaces
  163. become dots, and all other non-alphanumeric characters
  164. (except dots) become dashes, with runs of multiple
  165. dashes condensed to a single dash.
  166. :type version: string
  167. :returns: directory name
  168. :rtype: string"""
  169. name = name.replace('-', '_')
  170. return '-'.join([name, version]) + DISTINFO_EXT
  171. def get_distributions(self):
  172. """
  173. Provides an iterator that looks for distributions and returns
  174. :class:`InstalledDistribution` or
  175. :class:`EggInfoDistribution` instances for each one of them.
  176. :rtype: iterator of :class:`InstalledDistribution` and
  177. :class:`EggInfoDistribution` instances
  178. """
  179. if not self._cache_enabled:
  180. for dist in self._yield_distributions():
  181. yield dist
  182. else:
  183. self._generate_cache()
  184. for dist in self._cache.path.values():
  185. yield dist
  186. if self._include_egg:
  187. for dist in self._cache_egg.path.values():
  188. yield dist
  189. def get_distribution(self, name):
  190. """
  191. Looks for a named distribution on the path.
  192. This function only returns the first result found, as no more than one
  193. value is expected. If nothing is found, ``None`` is returned.
  194. :rtype: :class:`InstalledDistribution`, :class:`EggInfoDistribution`
  195. or ``None``
  196. """
  197. result = None
  198. name = name.lower()
  199. if not self._cache_enabled:
  200. for dist in self._yield_distributions():
  201. if dist.key == name:
  202. result = dist
  203. break
  204. else:
  205. self._generate_cache()
  206. if name in self._cache.name:
  207. result = self._cache.name[name][0]
  208. elif self._include_egg and name in self._cache_egg.name:
  209. result = self._cache_egg.name[name][0]
  210. return result
  211. def provides_distribution(self, name, version=None):
  212. """
  213. Iterates over all distributions to find which distributions provide *name*.
  214. If a *version* is provided, it will be used to filter the results.
  215. This function only returns the first result found, since no more than
  216. one values are expected. If the directory is not found, returns ``None``.
  217. :parameter version: a version specifier that indicates the version
  218. required, conforming to the format in ``PEP-345``
  219. :type name: string
  220. :type version: string
  221. """
  222. matcher = None
  223. if version is not None:
  224. try:
  225. matcher = self._scheme.matcher('%s (%s)' % (name, version))
  226. except ValueError:
  227. raise DistlibException('invalid name or version: %r, %r' %
  228. (name, version))
  229. for dist in self.get_distributions():
  230. # We hit a problem on Travis where enum34 was installed and doesn't
  231. # have a provides attribute ...
  232. if not hasattr(dist, 'provides'):
  233. logger.debug('No "provides": %s', dist)
  234. else:
  235. provided = dist.provides
  236. for p in provided:
  237. p_name, p_ver = parse_name_and_version(p)
  238. if matcher is None:
  239. if p_name == name:
  240. yield dist
  241. break
  242. else:
  243. if p_name == name and matcher.match(p_ver):
  244. yield dist
  245. break
  246. def get_file_path(self, name, relative_path):
  247. """
  248. Return the path to a resource file.
  249. """
  250. dist = self.get_distribution(name)
  251. if dist is None:
  252. raise LookupError('no distribution named %r found' % name)
  253. return dist.get_resource_path(relative_path)
  254. def get_exported_entries(self, category, name=None):
  255. """
  256. Return all of the exported entries in a particular category.
  257. :param category: The category to search for entries.
  258. :param name: If specified, only entries with that name are returned.
  259. """
  260. for dist in self.get_distributions():
  261. r = dist.exports
  262. if category in r:
  263. d = r[category]
  264. if name is not None:
  265. if name in d:
  266. yield d[name]
  267. else:
  268. for v in d.values():
  269. yield v
  270. class Distribution(object):
  271. """
  272. A base class for distributions, whether installed or from indexes.
  273. Either way, it must have some metadata, so that's all that's needed
  274. for construction.
  275. """
  276. build_time_dependency = False
  277. """
  278. Set to True if it's known to be only a build-time dependency (i.e.
  279. not needed after installation).
  280. """
  281. requested = False
  282. """A boolean that indicates whether the ``REQUESTED`` metadata file is
  283. present (in other words, whether the package was installed by user
  284. request or it was installed as a dependency)."""
  285. def __init__(self, metadata):
  286. """
  287. Initialise an instance.
  288. :param metadata: The instance of :class:`Metadata` describing this
  289. distribution.
  290. """
  291. self.metadata = metadata
  292. self.name = metadata.name
  293. self.key = self.name.lower() # for case-insensitive comparisons
  294. self.version = metadata.version
  295. self.locator = None
  296. self.digest = None
  297. self.extras = None # additional features requested
  298. self.context = None # environment marker overrides
  299. self.download_urls = set()
  300. self.digests = {}
  301. @property
  302. def source_url(self):
  303. """
  304. The source archive download URL for this distribution.
  305. """
  306. return self.metadata.source_url
  307. download_url = source_url # Backward compatibility
  308. @property
  309. def name_and_version(self):
  310. """
  311. A utility property which displays the name and version in parentheses.
  312. """
  313. return '%s (%s)' % (self.name, self.version)
  314. @property
  315. def provides(self):
  316. """
  317. A set of distribution names and versions provided by this distribution.
  318. :return: A set of "name (version)" strings.
  319. """
  320. plist = self.metadata.provides
  321. s = '%s (%s)' % (self.name, self.version)
  322. if s not in plist:
  323. plist.append(s)
  324. return plist
  325. def _get_requirements(self, req_attr):
  326. md = self.metadata
  327. logger.debug('Getting requirements from metadata %r', md.todict())
  328. reqts = getattr(md, req_attr)
  329. return set(md.get_requirements(reqts, extras=self.extras,
  330. env=self.context))
  331. @property
  332. def run_requires(self):
  333. return self._get_requirements('run_requires')
  334. @property
  335. def meta_requires(self):
  336. return self._get_requirements('meta_requires')
  337. @property
  338. def build_requires(self):
  339. return self._get_requirements('build_requires')
  340. @property
  341. def test_requires(self):
  342. return self._get_requirements('test_requires')
  343. @property
  344. def dev_requires(self):
  345. return self._get_requirements('dev_requires')
  346. def matches_requirement(self, req):
  347. """
  348. Say if this instance matches (fulfills) a requirement.
  349. :param req: The requirement to match.
  350. :rtype req: str
  351. :return: True if it matches, else False.
  352. """
  353. # Requirement may contain extras - parse to lose those
  354. # from what's passed to the matcher
  355. r = parse_requirement(req)
  356. scheme = get_scheme(self.metadata.scheme)
  357. try:
  358. matcher = scheme.matcher(r.requirement)
  359. except UnsupportedVersionError:
  360. # XXX compat-mode if cannot read the version
  361. logger.warning('could not read version %r - using name only',
  362. req)
  363. name = req.split()[0]
  364. matcher = scheme.matcher(name)
  365. name = matcher.key # case-insensitive
  366. result = False
  367. for p in self.provides:
  368. p_name, p_ver = parse_name_and_version(p)
  369. if p_name != name:
  370. continue
  371. try:
  372. result = matcher.match(p_ver)
  373. break
  374. except UnsupportedVersionError:
  375. pass
  376. return result
  377. def __repr__(self):
  378. """
  379. Return a textual representation of this instance,
  380. """
  381. if self.source_url:
  382. suffix = ' [%s]' % self.source_url
  383. else:
  384. suffix = ''
  385. return '<Distribution %s (%s)%s>' % (self.name, self.version, suffix)
  386. def __eq__(self, other):
  387. """
  388. See if this distribution is the same as another.
  389. :param other: The distribution to compare with. To be equal to one
  390. another. distributions must have the same type, name,
  391. version and source_url.
  392. :return: True if it is the same, else False.
  393. """
  394. if type(other) is not type(self):
  395. result = False
  396. else:
  397. result = (self.name == other.name and
  398. self.version == other.version and
  399. self.source_url == other.source_url)
  400. return result
  401. def __hash__(self):
  402. """
  403. Compute hash in a way which matches the equality test.
  404. """
  405. return hash(self.name) + hash(self.version) + hash(self.source_url)
  406. class BaseInstalledDistribution(Distribution):
  407. """
  408. This is the base class for installed distributions (whether PEP 376 or
  409. legacy).
  410. """
  411. hasher = None
  412. def __init__(self, metadata, path, env=None):
  413. """
  414. Initialise an instance.
  415. :param metadata: An instance of :class:`Metadata` which describes the
  416. distribution. This will normally have been initialised
  417. from a metadata file in the ``path``.
  418. :param path: The path of the ``.dist-info`` or ``.egg-info``
  419. directory for the distribution.
  420. :param env: This is normally the :class:`DistributionPath`
  421. instance where this distribution was found.
  422. """
  423. super(BaseInstalledDistribution, self).__init__(metadata)
  424. self.path = path
  425. self.dist_path = env
  426. def get_hash(self, data, hasher=None):
  427. """
  428. Get the hash of some data, using a particular hash algorithm, if
  429. specified.
  430. :param data: The data to be hashed.
  431. :type data: bytes
  432. :param hasher: The name of a hash implementation, supported by hashlib,
  433. or ``None``. Examples of valid values are ``'sha1'``,
  434. ``'sha224'``, ``'sha384'``, '``sha256'``, ``'md5'`` and
  435. ``'sha512'``. If no hasher is specified, the ``hasher``
  436. attribute of the :class:`InstalledDistribution` instance
  437. is used. If the hasher is determined to be ``None``, MD5
  438. is used as the hashing algorithm.
  439. :returns: The hash of the data. If a hasher was explicitly specified,
  440. the returned hash will be prefixed with the specified hasher
  441. followed by '='.
  442. :rtype: str
  443. """
  444. if hasher is None:
  445. hasher = self.hasher
  446. if hasher is None:
  447. hasher = hashlib.md5
  448. prefix = ''
  449. else:
  450. hasher = getattr(hashlib, hasher)
  451. prefix = '%s=' % self.hasher
  452. digest = hasher(data).digest()
  453. digest = base64.urlsafe_b64encode(digest).rstrip(b'=').decode('ascii')
  454. return '%s%s' % (prefix, digest)
  455. class InstalledDistribution(BaseInstalledDistribution):
  456. """
  457. Created with the *path* of the ``.dist-info`` directory provided to the
  458. constructor. It reads the metadata contained in ``pydist.json`` when it is
  459. instantiated., or uses a passed in Metadata instance (useful for when
  460. dry-run mode is being used).
  461. """
  462. hasher = 'sha256'
  463. def __init__(self, path, metadata=None, env=None):
  464. self.modules = []
  465. self.finder = finder = resources.finder_for_path(path)
  466. if finder is None:
  467. raise ValueError('finder unavailable for %s' % path)
  468. if env and env._cache_enabled and path in env._cache.path:
  469. metadata = env._cache.path[path].metadata
  470. elif metadata is None:
  471. r = finder.find(METADATA_FILENAME)
  472. # Temporary - for Wheel 0.23 support
  473. if r is None:
  474. r = finder.find(WHEEL_METADATA_FILENAME)
  475. # Temporary - for legacy support
  476. if r is None:
  477. r = finder.find(LEGACY_METADATA_FILENAME)
  478. if r is None:
  479. raise ValueError('no %s found in %s' % (METADATA_FILENAME,
  480. path))
  481. with contextlib.closing(r.as_stream()) as stream:
  482. metadata = Metadata(fileobj=stream, scheme='legacy')
  483. super(InstalledDistribution, self).__init__(metadata, path, env)
  484. if env and env._cache_enabled:
  485. env._cache.add(self)
  486. r = finder.find('REQUESTED')
  487. self.requested = r is not None
  488. p = os.path.join(path, 'top_level.txt')
  489. if os.path.exists(p):
  490. with open(p, 'rb') as f:
  491. data = f.read().decode('utf-8')
  492. self.modules = data.splitlines()
  493. def __repr__(self):
  494. return '<InstalledDistribution %r %s at %r>' % (
  495. self.name, self.version, self.path)
  496. def __str__(self):
  497. return "%s %s" % (self.name, self.version)
  498. def _get_records(self):
  499. """
  500. Get the list of installed files for the distribution
  501. :return: A list of tuples of path, hash and size. Note that hash and
  502. size might be ``None`` for some entries. The path is exactly
  503. as stored in the file (which is as in PEP 376).
  504. """
  505. results = []
  506. r = self.get_distinfo_resource('RECORD')
  507. with contextlib.closing(r.as_stream()) as stream:
  508. with CSVReader(stream=stream) as record_reader:
  509. # Base location is parent dir of .dist-info dir
  510. #base_location = os.path.dirname(self.path)
  511. #base_location = os.path.abspath(base_location)
  512. for row in record_reader:
  513. missing = [None for i in range(len(row), 3)]
  514. path, checksum, size = row + missing
  515. #if not os.path.isabs(path):
  516. # path = path.replace('/', os.sep)
  517. # path = os.path.join(base_location, path)
  518. results.append((path, checksum, size))
  519. return results
  520. @cached_property
  521. def exports(self):
  522. """
  523. Return the information exported by this distribution.
  524. :return: A dictionary of exports, mapping an export category to a dict
  525. of :class:`ExportEntry` instances describing the individual
  526. export entries, and keyed by name.
  527. """
  528. result = {}
  529. r = self.get_distinfo_resource(EXPORTS_FILENAME)
  530. if r:
  531. result = self.read_exports()
  532. return result
  533. def read_exports(self):
  534. """
  535. Read exports data from a file in .ini format.
  536. :return: A dictionary of exports, mapping an export category to a list
  537. of :class:`ExportEntry` instances describing the individual
  538. export entries.
  539. """
  540. result = {}
  541. r = self.get_distinfo_resource(EXPORTS_FILENAME)
  542. if r:
  543. with contextlib.closing(r.as_stream()) as stream:
  544. result = read_exports(stream)
  545. return result
  546. def write_exports(self, exports):
  547. """
  548. Write a dictionary of exports to a file in .ini format.
  549. :param exports: A dictionary of exports, mapping an export category to
  550. a list of :class:`ExportEntry` instances describing the
  551. individual export entries.
  552. """
  553. rf = self.get_distinfo_file(EXPORTS_FILENAME)
  554. with open(rf, 'w') as f:
  555. write_exports(exports, f)
  556. def get_resource_path(self, relative_path):
  557. """
  558. NOTE: This API may change in the future.
  559. Return the absolute path to a resource file with the given relative
  560. path.
  561. :param relative_path: The path, relative to .dist-info, of the resource
  562. of interest.
  563. :return: The absolute path where the resource is to be found.
  564. """
  565. r = self.get_distinfo_resource('RESOURCES')
  566. with contextlib.closing(r.as_stream()) as stream:
  567. with CSVReader(stream=stream) as resources_reader:
  568. for relative, destination in resources_reader:
  569. if relative == relative_path:
  570. return destination
  571. raise KeyError('no resource file with relative path %r '
  572. 'is installed' % relative_path)
  573. def list_installed_files(self):
  574. """
  575. Iterates over the ``RECORD`` entries and returns a tuple
  576. ``(path, hash, size)`` for each line.
  577. :returns: iterator of (path, hash, size)
  578. """
  579. for result in self._get_records():
  580. yield result
  581. def write_installed_files(self, paths, prefix, dry_run=False):
  582. """
  583. Writes the ``RECORD`` file, using the ``paths`` iterable passed in. Any
  584. existing ``RECORD`` file is silently overwritten.
  585. prefix is used to determine when to write absolute paths.
  586. """
  587. prefix = os.path.join(prefix, '')
  588. base = os.path.dirname(self.path)
  589. base_under_prefix = base.startswith(prefix)
  590. base = os.path.join(base, '')
  591. record_path = self.get_distinfo_file('RECORD')
  592. logger.info('creating %s', record_path)
  593. if dry_run:
  594. return None
  595. with CSVWriter(record_path) as writer:
  596. for path in paths:
  597. if os.path.isdir(path) or path.endswith(('.pyc', '.pyo')):
  598. # do not put size and hash, as in PEP-376
  599. hash_value = size = ''
  600. else:
  601. size = '%d' % os.path.getsize(path)
  602. with open(path, 'rb') as fp:
  603. hash_value = self.get_hash(fp.read())
  604. if path.startswith(base) or (base_under_prefix and
  605. path.startswith(prefix)):
  606. path = os.path.relpath(path, base)
  607. writer.writerow((path, hash_value, size))
  608. # add the RECORD file itself
  609. if record_path.startswith(base):
  610. record_path = os.path.relpath(record_path, base)
  611. writer.writerow((record_path, '', ''))
  612. return record_path
  613. def check_installed_files(self):
  614. """
  615. Checks that the hashes and sizes of the files in ``RECORD`` are
  616. matched by the files themselves. Returns a (possibly empty) list of
  617. mismatches. Each entry in the mismatch list will be a tuple consisting
  618. of the path, 'exists', 'size' or 'hash' according to what didn't match
  619. (existence is checked first, then size, then hash), the expected
  620. value and the actual value.
  621. """
  622. mismatches = []
  623. base = os.path.dirname(self.path)
  624. record_path = self.get_distinfo_file('RECORD')
  625. for path, hash_value, size in self.list_installed_files():
  626. if not os.path.isabs(path):
  627. path = os.path.join(base, path)
  628. if path == record_path:
  629. continue
  630. if not os.path.exists(path):
  631. mismatches.append((path, 'exists', True, False))
  632. elif os.path.isfile(path):
  633. actual_size = str(os.path.getsize(path))
  634. if size and actual_size != size:
  635. mismatches.append((path, 'size', size, actual_size))
  636. elif hash_value:
  637. if '=' in hash_value:
  638. hasher = hash_value.split('=', 1)[0]
  639. else:
  640. hasher = None
  641. with open(path, 'rb') as f:
  642. actual_hash = self.get_hash(f.read(), hasher)
  643. if actual_hash != hash_value:
  644. mismatches.append((path, 'hash', hash_value, actual_hash))
  645. return mismatches
  646. @cached_property
  647. def shared_locations(self):
  648. """
  649. A dictionary of shared locations whose keys are in the set 'prefix',
  650. 'purelib', 'platlib', 'scripts', 'headers', 'data' and 'namespace'.
  651. The corresponding value is the absolute path of that category for
  652. this distribution, and takes into account any paths selected by the
  653. user at installation time (e.g. via command-line arguments). In the
  654. case of the 'namespace' key, this would be a list of absolute paths
  655. for the roots of namespace packages in this distribution.
  656. The first time this property is accessed, the relevant information is
  657. read from the SHARED file in the .dist-info directory.
  658. """
  659. result = {}
  660. shared_path = os.path.join(self.path, 'SHARED')
  661. if os.path.isfile(shared_path):
  662. with codecs.open(shared_path, 'r', encoding='utf-8') as f:
  663. lines = f.read().splitlines()
  664. for line in lines:
  665. key, value = line.split('=', 1)
  666. if key == 'namespace':
  667. result.setdefault(key, []).append(value)
  668. else:
  669. result[key] = value
  670. return result
  671. def write_shared_locations(self, paths, dry_run=False):
  672. """
  673. Write shared location information to the SHARED file in .dist-info.
  674. :param paths: A dictionary as described in the documentation for
  675. :meth:`shared_locations`.
  676. :param dry_run: If True, the action is logged but no file is actually
  677. written.
  678. :return: The path of the file written to.
  679. """
  680. shared_path = os.path.join(self.path, 'SHARED')
  681. logger.info('creating %s', shared_path)
  682. if dry_run:
  683. return None
  684. lines = []
  685. for key in ('prefix', 'lib', 'headers', 'scripts', 'data'):
  686. path = paths[key]
  687. if os.path.isdir(paths[key]):
  688. lines.append('%s=%s' % (key, path))
  689. for ns in paths.get('namespace', ()):
  690. lines.append('namespace=%s' % ns)
  691. with codecs.open(shared_path, 'w', encoding='utf-8') as f:
  692. f.write('\n'.join(lines))
  693. return shared_path
  694. def get_distinfo_resource(self, path):
  695. if path not in DIST_FILES:
  696. raise DistlibException('invalid path for a dist-info file: '
  697. '%r at %r' % (path, self.path))
  698. finder = resources.finder_for_path(self.path)
  699. if finder is None:
  700. raise DistlibException('Unable to get a finder for %s' % self.path)
  701. return finder.find(path)
  702. def get_distinfo_file(self, path):
  703. """
  704. Returns a path located under the ``.dist-info`` directory. Returns a
  705. string representing the path.
  706. :parameter path: a ``'/'``-separated path relative to the
  707. ``.dist-info`` directory or an absolute path;
  708. If *path* is an absolute path and doesn't start
  709. with the ``.dist-info`` directory path,
  710. a :class:`DistlibException` is raised
  711. :type path: str
  712. :rtype: str
  713. """
  714. # Check if it is an absolute path # XXX use relpath, add tests
  715. if path.find(os.sep) >= 0:
  716. # it's an absolute path?
  717. distinfo_dirname, path = path.split(os.sep)[-2:]
  718. if distinfo_dirname != self.path.split(os.sep)[-1]:
  719. raise DistlibException(
  720. 'dist-info file %r does not belong to the %r %s '
  721. 'distribution' % (path, self.name, self.version))
  722. # The file must be relative
  723. if path not in DIST_FILES:
  724. raise DistlibException('invalid path for a dist-info file: '
  725. '%r at %r' % (path, self.path))
  726. return os.path.join(self.path, path)
  727. def list_distinfo_files(self):
  728. """
  729. Iterates over the ``RECORD`` entries and returns paths for each line if
  730. the path is pointing to a file located in the ``.dist-info`` directory
  731. or one of its subdirectories.
  732. :returns: iterator of paths
  733. """
  734. base = os.path.dirname(self.path)
  735. for path, checksum, size in self._get_records():
  736. # XXX add separator or use real relpath algo
  737. if not os.path.isabs(path):
  738. path = os.path.join(base, path)
  739. if path.startswith(self.path):
  740. yield path
  741. def __eq__(self, other):
  742. return (isinstance(other, InstalledDistribution) and
  743. self.path == other.path)
  744. # See http://docs.python.org/reference/datamodel#object.__hash__
  745. __hash__ = object.__hash__
  746. class EggInfoDistribution(BaseInstalledDistribution):
  747. """Created with the *path* of the ``.egg-info`` directory or file provided
  748. to the constructor. It reads the metadata contained in the file itself, or
  749. if the given path happens to be a directory, the metadata is read from the
  750. file ``PKG-INFO`` under that directory."""
  751. requested = True # as we have no way of knowing, assume it was
  752. shared_locations = {}
  753. def __init__(self, path, env=None):
  754. def set_name_and_version(s, n, v):
  755. s.name = n
  756. s.key = n.lower() # for case-insensitive comparisons
  757. s.version = v
  758. self.path = path
  759. self.dist_path = env
  760. if env and env._cache_enabled and path in env._cache_egg.path:
  761. metadata = env._cache_egg.path[path].metadata
  762. set_name_and_version(self, metadata.name, metadata.version)
  763. else:
  764. metadata = self._get_metadata(path)
  765. # Need to be set before caching
  766. set_name_and_version(self, metadata.name, metadata.version)
  767. if env and env._cache_enabled:
  768. env._cache_egg.add(self)
  769. super(EggInfoDistribution, self).__init__(metadata, path, env)
  770. def _get_metadata(self, path):
  771. requires = None
  772. def parse_requires_data(data):
  773. """Create a list of dependencies from a requires.txt file.
  774. *data*: the contents of a setuptools-produced requires.txt file.
  775. """
  776. reqs = []
  777. lines = data.splitlines()
  778. for line in lines:
  779. line = line.strip()
  780. if line.startswith('['):
  781. logger.warning('Unexpected line: quitting requirement scan: %r',
  782. line)
  783. break
  784. r = parse_requirement(line)
  785. if not r:
  786. logger.warning('Not recognised as a requirement: %r', line)
  787. continue
  788. if r.extras:
  789. logger.warning('extra requirements in requires.txt are '
  790. 'not supported')
  791. if not r.constraints:
  792. reqs.append(r.name)
  793. else:
  794. cons = ', '.join('%s%s' % c for c in r.constraints)
  795. reqs.append('%s (%s)' % (r.name, cons))
  796. return reqs
  797. def parse_requires_path(req_path):
  798. """Create a list of dependencies from a requires.txt file.
  799. *req_path*: the path to a setuptools-produced requires.txt file.
  800. """
  801. reqs = []
  802. try:
  803. with codecs.open(req_path, 'r', 'utf-8') as fp:
  804. reqs = parse_requires_data(fp.read())
  805. except IOError:
  806. pass
  807. return reqs
  808. tl_path = tl_data = None
  809. if path.endswith('.egg'):
  810. if os.path.isdir(path):
  811. p = os.path.join(path, 'EGG-INFO')
  812. meta_path = os.path.join(p, 'PKG-INFO')
  813. metadata = Metadata(path=meta_path, scheme='legacy')
  814. req_path = os.path.join(p, 'requires.txt')
  815. tl_path = os.path.join(p, 'top_level.txt')
  816. requires = parse_requires_path(req_path)
  817. else:
  818. # FIXME handle the case where zipfile is not available
  819. zipf = zipimport.zipimporter(path)
  820. fileobj = StringIO(
  821. zipf.get_data('EGG-INFO/PKG-INFO').decode('utf8'))
  822. metadata = Metadata(fileobj=fileobj, scheme='legacy')
  823. try:
  824. data = zipf.get_data('EGG-INFO/requires.txt')
  825. tl_data = zipf.get_data('EGG-INFO/top_level.txt').decode('utf-8')
  826. requires = parse_requires_data(data.decode('utf-8'))
  827. except IOError:
  828. requires = None
  829. elif path.endswith('.egg-info'):
  830. if os.path.isdir(path):
  831. req_path = os.path.join(path, 'requires.txt')
  832. requires = parse_requires_path(req_path)
  833. path = os.path.join(path, 'PKG-INFO')
  834. tl_path = os.path.join(path, 'top_level.txt')
  835. metadata = Metadata(path=path, scheme='legacy')
  836. else:
  837. raise DistlibException('path must end with .egg-info or .egg, '
  838. 'got %r' % path)
  839. if requires:
  840. metadata.add_requirements(requires)
  841. # look for top-level modules in top_level.txt, if present
  842. if tl_data is None:
  843. if tl_path is not None and os.path.exists(tl_path):
  844. with open(tl_path, 'rb') as f:
  845. tl_data = f.read().decode('utf-8')
  846. if not tl_data:
  847. tl_data = []
  848. else:
  849. tl_data = tl_data.splitlines()
  850. self.modules = tl_data
  851. return metadata
  852. def __repr__(self):
  853. return '<EggInfoDistribution %r %s at %r>' % (
  854. self.name, self.version, self.path)
  855. def __str__(self):
  856. return "%s %s" % (self.name, self.version)
  857. def check_installed_files(self):
  858. """
  859. Checks that the hashes and sizes of the files in ``RECORD`` are
  860. matched by the files themselves. Returns a (possibly empty) list of
  861. mismatches. Each entry in the mismatch list will be a tuple consisting
  862. of the path, 'exists', 'size' or 'hash' according to what didn't match
  863. (existence is checked first, then size, then hash), the expected
  864. value and the actual value.
  865. """
  866. mismatches = []
  867. record_path = os.path.join(self.path, 'installed-files.txt')
  868. if os.path.exists(record_path):
  869. for path, _, _ in self.list_installed_files():
  870. if path == record_path:
  871. continue
  872. if not os.path.exists(path):
  873. mismatches.append((path, 'exists', True, False))
  874. return mismatches
  875. def list_installed_files(self):
  876. """
  877. Iterates over the ``installed-files.txt`` entries and returns a tuple
  878. ``(path, hash, size)`` for each line.
  879. :returns: a list of (path, hash, size)
  880. """
  881. def _md5(path):
  882. f = open(path, 'rb')
  883. try:
  884. content = f.read()
  885. finally:
  886. f.close()
  887. return hashlib.md5(content).hexdigest()
  888. def _size(path):
  889. return os.stat(path).st_size
  890. record_path = os.path.join(self.path, 'installed-files.txt')
  891. result = []
  892. if os.path.exists(record_path):
  893. with codecs.open(record_path, 'r', encoding='utf-8') as f:
  894. for line in f:
  895. line = line.strip()
  896. p = os.path.normpath(os.path.join(self.path, line))
  897. # "./" is present as a marker between installed files
  898. # and installation metadata files
  899. if not os.path.exists(p):
  900. logger.warning('Non-existent file: %s', p)
  901. if p.endswith(('.pyc', '.pyo')):
  902. continue
  903. #otherwise fall through and fail
  904. if not os.path.isdir(p):
  905. result.append((p, _md5(p), _size(p)))
  906. result.append((record_path, None, None))
  907. return result
  908. def list_distinfo_files(self, absolute=False):
  909. """
  910. Iterates over the ``installed-files.txt`` entries and returns paths for
  911. each line if the path is pointing to a file located in the
  912. ``.egg-info`` directory or one of its subdirectories.
  913. :parameter absolute: If *absolute* is ``True``, each returned path is
  914. transformed into a local absolute path. Otherwise the
  915. raw value from ``installed-files.txt`` is returned.
  916. :type absolute: boolean
  917. :returns: iterator of paths
  918. """
  919. record_path = os.path.join(self.path, 'installed-files.txt')
  920. if os.path.exists(record_path):
  921. skip = True
  922. with codecs.open(record_path, 'r', encoding='utf-8') as f:
  923. for line in f:
  924. line = line.strip()
  925. if line == './':
  926. skip = False
  927. continue
  928. if not skip:
  929. p = os.path.normpath(os.path.join(self.path, line))
  930. if p.startswith(self.path):
  931. if absolute:
  932. yield p
  933. else:
  934. yield line
  935. def __eq__(self, other):
  936. return (isinstance(other, EggInfoDistribution) and
  937. self.path == other.path)
  938. # See http://docs.python.org/reference/datamodel#object.__hash__
  939. __hash__ = object.__hash__
  940. new_dist_class = InstalledDistribution
  941. old_dist_class = EggInfoDistribution
  942. class DependencyGraph(object):
  943. """
  944. Represents a dependency graph between distributions.
  945. The dependency relationships are stored in an ``adjacency_list`` that maps
  946. distributions to a list of ``(other, label)`` tuples where ``other``
  947. is a distribution and the edge is labeled with ``label`` (i.e. the version
  948. specifier, if such was provided). Also, for more efficient traversal, for
  949. every distribution ``x``, a list of predecessors is kept in
  950. ``reverse_list[x]``. An edge from distribution ``a`` to
  951. distribution ``b`` means that ``a`` depends on ``b``. If any missing
  952. dependencies are found, they are stored in ``missing``, which is a
  953. dictionary that maps distributions to a list of requirements that were not
  954. provided by any other distributions.
  955. """
  956. def __init__(self):
  957. self.adjacency_list = {}
  958. self.reverse_list = {}
  959. self.missing = {}
  960. def add_distribution(self, distribution):
  961. """Add the *distribution* to the graph.
  962. :type distribution: :class:`distutils2.database.InstalledDistribution`
  963. or :class:`distutils2.database.EggInfoDistribution`
  964. """
  965. self.adjacency_list[distribution] = []
  966. self.reverse_list[distribution] = []
  967. #self.missing[distribution] = []
  968. def add_edge(self, x, y, label=None):
  969. """Add an edge from distribution *x* to distribution *y* with the given
  970. *label*.
  971. :type x: :class:`distutils2.database.InstalledDistribution` or
  972. :class:`distutils2.database.EggInfoDistribution`
  973. :type y: :class:`distutils2.database.InstalledDistribution` or
  974. :class:`distutils2.database.EggInfoDistribution`
  975. :type label: ``str`` or ``None``
  976. """
  977. self.adjacency_list[x].append((y, label))
  978. # multiple edges are allowed, so be careful
  979. if x not in self.reverse_list[y]:
  980. self.reverse_list[y].append(x)
  981. def add_missing(self, distribution, requirement):
  982. """
  983. Add a missing *requirement* for the given *distribution*.
  984. :type distribution: :class:`distutils2.database.InstalledDistribution`
  985. or :class:`distutils2.database.EggInfoDistribution`
  986. :type requirement: ``str``
  987. """
  988. logger.debug('%s missing %r', distribution, requirement)
  989. self.missing.setdefault(distribution, []).append(requirement)
  990. def _repr_dist(self, dist):
  991. return '%s %s' % (dist.name, dist.version)
  992. def repr_node(self, dist, level=1):
  993. """Prints only a subgraph"""
  994. output = [self._repr_dist(dist)]
  995. for other, label in self.adjacency_list[dist]:
  996. dist = self._repr_dist(other)
  997. if label is not None:
  998. dist = '%s [%s]' % (dist, label)
  999. output.append(' ' * level + str(dist))
  1000. suboutput = self.repr_node(other, level + 1)
  1001. subs = suboutput.split('\n')
  1002. output.extend(subs[1:])
  1003. return '\n'.join(output)
  1004. def to_dot(self, f, skip_disconnected=True):
  1005. """Writes a DOT output for the graph to the provided file *f*.
  1006. If *skip_disconnected* is set to ``True``, then all distributions
  1007. that are not dependent on any other distribution are skipped.
  1008. :type f: has to support ``file``-like operations
  1009. :type skip_disconnected: ``bool``
  1010. """
  1011. disconnected = []
  1012. f.write("digraph dependencies {\n")
  1013. for dist, adjs in self.adjacency_list.items():
  1014. if len(adjs) == 0 and not skip_disconnected:
  1015. disconnected.append(dist)
  1016. for other, label in adjs:
  1017. if not label is None:
  1018. f.write('"%s" -> "%s" [label="%s"]\n' %
  1019. (dist.name, other.name, label))
  1020. else:
  1021. f.write('"%s" -> "%s"\n' % (dist.name, other.name))
  1022. if not skip_disconnected and len(disconnected) > 0:
  1023. f.write('subgraph disconnected {\n')
  1024. f.write('label = "Disconnected"\n')
  1025. f.write('bgcolor = red\n')
  1026. for dist in disconnected:
  1027. f.write('"%s"' % dist.name)
  1028. f.write('\n')
  1029. f.write('}\n')
  1030. f.write('}\n')
  1031. def topological_sort(self):
  1032. """
  1033. Perform a topological sort of the graph.
  1034. :return: A tuple, the first element of which is a topologically sorted
  1035. list of distributions, and the second element of which is a
  1036. list of distributions that cannot be sorted because they have
  1037. circular dependencies and so form a cycle.
  1038. """
  1039. result = []
  1040. # Make a shallow copy of the adjacency list
  1041. alist = {}
  1042. for k, v in self.adjacency_list.items():
  1043. alist[k] = v[:]
  1044. while True:
  1045. # See what we can remove in this run
  1046. to_remove = []
  1047. for k, v in list(alist.items())[:]:
  1048. if not v:
  1049. to_remove.append(k)
  1050. del alist[k]
  1051. if not to_remove:
  1052. # What's left in alist (if anything) is a cycle.
  1053. break
  1054. # Remove from the adjacency list of others
  1055. for k, v in alist.items():
  1056. alist[k] = [(d, r) for d, r in v if d not in to_remove]
  1057. logger.debug('Moving to result: %s',
  1058. ['%s (%s)' % (d.name, d.version) for d in to_remove])
  1059. result.extend(to_remove)
  1060. return result, list(alist.keys())
  1061. def __repr__(self):
  1062. """Representation of the graph"""
  1063. output = []
  1064. for dist, adjs in self.adjacency_list.items():
  1065. output.append(self.repr_node(dist))
  1066. return '\n'.join(output)
  1067. def make_graph(dists, scheme='default'):
  1068. """Makes a dependency graph from the given distributions.
  1069. :parameter dists: a list of distributions
  1070. :type dists: list of :class:`distutils2.database.InstalledDistribution` and
  1071. :class:`distutils2.database.EggInfoDistribution` instances
  1072. :rtype: a :class:`DependencyGraph` instance
  1073. """
  1074. scheme = get_scheme(scheme)
  1075. graph = DependencyGraph()
  1076. provided = {} # maps names to lists of (version, dist) tuples
  1077. # first, build the graph and find out what's provided
  1078. for dist in dists:
  1079. graph.add_distribution(dist)
  1080. for p in dist.provides:
  1081. name, version = parse_name_and_version(p)
  1082. logger.debug('Add to provided: %s, %s, %s', name, version, dist)
  1083. provided.setdefault(name, []).append((version, dist))
  1084. # now make the edges
  1085. for dist in dists:
  1086. requires = (dist.run_requires | dist.meta_requires |
  1087. dist.build_requires | dist.dev_requires)
  1088. for req in requires:
  1089. try:
  1090. matcher = scheme.matcher(req)
  1091. except UnsupportedVersionError:
  1092. # XXX compat-mode if cannot read the version
  1093. logger.warning('could not read version %r - using name only',
  1094. req)
  1095. name = req.split()[0]
  1096. matcher = scheme.matcher(name)
  1097. name = matcher.key # case-insensitive
  1098. matched = False
  1099. if name in provided:
  1100. for version, provider in provided[name]:
  1101. try:
  1102. match = matcher.match(version)
  1103. except UnsupportedVersionError:
  1104. match = False
  1105. if match:
  1106. graph.add_edge(dist, provider, req)
  1107. matched = True
  1108. break
  1109. if not matched:
  1110. graph.add_missing(dist, req)
  1111. return graph
  1112. def get_dependent_dists(dists, dist):
  1113. """Recursively generate a list of distributions from *dists* that are
  1114. dependent on *dist*.
  1115. :param dists: a list of distributions
  1116. :param dist: a distribution, member of *dists* for which we are interested
  1117. """
  1118. if dist not in dists:
  1119. raise DistlibException('given distribution %r is not a member '
  1120. 'of the list' % dist.name)
  1121. graph = make_graph(dists)
  1122. dep = [dist] # dependent distributions
  1123. todo = graph.reverse_list[dist] # list of nodes we should inspect
  1124. while todo:
  1125. d = todo.pop()
  1126. dep.append(d)
  1127. for succ in graph.reverse_list[d]:
  1128. if succ not in dep:
  1129. todo.append(succ)
  1130. dep.pop(0) # remove dist from dep, was there to prevent infinite loops
  1131. return dep
  1132. def get_required_dists(dists, dist):
  1133. """Recursively generate a list of distributions from *dists* that are
  1134. required by *dist*.
  1135. :param dists: a list of distributions
  1136. :param dist: a distribution, member of *dists* for which we are interested
  1137. """
  1138. if dist not in dists:
  1139. raise DistlibException('given distribution %r is not a member '
  1140. 'of the list' % dist.name)
  1141. graph = make_graph(dists)
  1142. req = [] # required distributions
  1143. todo = graph.adjacency_list[dist] # list of nodes we should inspect
  1144. while todo:
  1145. d = todo.pop()[0]
  1146. req.append(d)
  1147. for pred in graph.adjacency_list[d]:
  1148. if pred not in req:
  1149. todo.append(pred)
  1150. return req
  1151. def make_dist(name, version, **kwargs):
  1152. """
  1153. A convenience method for making a dist given just a name and version.
  1154. """
  1155. summary = kwargs.pop('summary', 'Placeholder for summary')
  1156. md = Metadata(**kwargs)
  1157. md.name = name
  1158. md.version = version
  1159. md.summary = summary or 'Placeholder for summary'
  1160. return Distribution(md)